#!/usr/bin/env/ python
# -*- coding:utf-8 -*-

__author__ = 'shouke'

import configparser
import threading
import time
import  re


from influxdb import InfluxDBClient
from mysshclient import MySSHClient
from log import logger


class Mornitor:
    def __init__(self, interval=1, count=1): # interval:指定采样时间间隔  count:指定采样次数
        try:
            self.inteval = interval
            self.count = count

            logger.info('正在读取influxdb数据库初始化配置')
            config = configparser.ConfigParser()
            config.read('./conf/influxDB.conf', encoding='utf-8')
            influxdb_host = config['INFLUXDB']['influxdb_host']
            influxdb_port = int(config['INFLUXDB']['influxdb_port'])

            self.influx_db_client = InfluxDBClient(influxdb_host, influxdb_port, timeout=10)
            self.database_list = self.influx_db_client.get_list_database()
            self.database_list = [dic_item['name'] for dic_item in self.database_list]
            logger.info('获取到的初始数据库列表：%s' % self.database_list)


            self.host_config = configparser.ConfigParser()
            self.host_config.read('./conf/host_config.conf', encoding='utf-8')  # 获取配置信息中，主机节点列表
            self.host_list = self.host_config.sections()
            target_host_set = self.parse_target_host_conf() # 获取需要监控的主机
            self.target_host_set = set(self.host_list) & target_host_set

            config.clear()
            config.read('./conf/bastion_host_config.conf', encoding='utf-8')
            section_list = config.sections() # 堡垒机ip地址列表
            self.bashtion_target_host_dic = {} # 存放需要通过堡垒机访问的目标主机，格式 {'堡垒机IP':['目标IP1', '目标IP2']}
            self.target_bashtion_host_dic = {} # 存放需要通过堡垒机访问的目标主机，格式 {'目标IP1':'堡垒机', '目标IP2':'堡垒机'}
            for section in section_list:
                option_list = config.options(section)
                bastion_host = section.strip().replace(' ', '')

                host_set = set() # 存放需要通过堡垒机访问的目标IP
                for option in option_list:
                    target_host = config[section][option]
                    target_host = target_host.strip().replace(' ', '')
                    self.bashtion_target_host_dic[target_host] = bastion_host
                    host_set.add(target_host)
                host_set = host_set & target_host_set
                self.bashtion_target_host_dic[bastion_host] = list(host_set)

            self.target_host_filter = {} # 存放目标主机过滤器 格式 {'目标IP'：'过滤器1', '过滤器2'}
            config.clear()
            config.read('./conf/host_filter.conf', encoding='utf-8')
            option_list = config.options('HOSTFILTER')
            for option in option_list:
                target_host = option.strip().replace(' ', '')
                self.target_host_filter[target_host] = config['HOSTFILTER'][option].strip().strip(',').replace(' ', '').split(',')

            config.clear()
            config.read('./conf/cgroup_path.conf', encoding='utf-8')
            option_list = config.options('CGROUPPATH')
            cgroup_path = {}
            for option in option_list:
                cgroup_path[option] = config['CGROUPPATH'][option]
            if cgroup_path == {}:
                logger.error('程序初始化操作失败：%s' % '未配置cgroup 路径，请检查cgroup_path.conf')
                exit()

            self.collector_sh_script = []
            with open('./conf/collector.sh', 'r', encoding='utf-8') as f:
                line = f.readline()
                while line:
                    self.collector_sh_script.append(line)
                    line = f.readline()
                    line = line.replace('{cpu_path}', cgroup_path['cpu_path']).replace('{cpuacct_path}', cgroup_path['cpuacct_path'])
                    line = line.replace('{memory_path}', cgroup_path['memory_path']).replace('{blkio_path}', cgroup_path['blkio_path'])
        except Exception as e:
            logger.error('程序初始化操作失败：%s' % e)
            exit()

    def parse_target_host_conf(self):
        target_host_list = set()
        with open('./conf/target_host_for_monitor.conf', encoding='utf-8') as f:
            line = f.readline()
            while line:
                line = line.strip()
                if line.startswith('#'): # 注释,跳过
                    line = f.readline()
                    continue
                elif re.findall('\s*[\d]+\.[\d]+\.[\d]+\.[\d]+\s*$', line): # 说明是主机ip,形如 192.168.1.21
                    target_host_list.add(line)
                line = f.readline()
        return target_host_list

    def exec_monitor(self, exclustion):
        def monitor(target_host, ssh_client, bastion_host=''):
            nonlocal file_for_failure
            logger.info('开始采集性能数据')

            if bastion_host == '':
                # command = "export LC_TIME=\"POSIX\";sar -u ALL -P ALL -q -w -r -S -W -B -pd -b -n EDEV -n DEV %s %s" % (self.inteval, self.count)
                command = "export LC_ALL=\"C\";sar -u ALL -P ALL -q -w -r -S -W -B -pd -b -n EDEV -n DEV %s %s" % (self.inteval, self.count)
            else:
                # command = "export LC_ALL=\"C\";sar -u ALL -P ALL -q -w -r -S -W -B -b -n EDEV -n DEV %s %s" % (self.inteval, self.count)
                command = 'sh collector.sh %s %s' % (self.inteval, self.count)
            exclustion_dic = {'onecpu':'-P ALL', 'queue':'-q', 'proc':'-w',  'mem':'-r', 'swap':'-W', 'swapspace':'-S',
                              'deviotps':'-b', 'netdev':'-n DEV', 'enetdev':'-n EDEV', 'disk': '-pd', 'paging':'-B'} # 不监控项目

            for item in exclustion:
                temp = item.lower()
                if temp in exclustion_dic.keys():
                    command = command.replace(exclustion_dic[temp], '')

            # 针对单台主机的过滤
            temp_host_list = list(self.target_host_filter.keys())
            if target_host in temp_host_list:
                for item in self.target_host_filter[target_host]:
                    temp = item.lower()
                    if temp in exclustion_dic.keys():
                        command = command.replace(exclustion_dic[temp], '')

            if bastion_host == '':
                ssh_client.exec_command(command, target_host, bastion_host)
            else:
                ssh_client.exec_command_in_docker_container(command, target_host, bastion_host, self.collector_sh_script)
            ssh_client.close()

        def parse_data(tag):
            nonlocal file_for_failure
            data_group_regex = re.compile('^\d\d:\d\d:\d\d[\s|@]')
            while len(MySSHClient.data_queue):
                host_for_data, run_time, data_group_list = MySSHClient.data_queue.popleft() # 不能popleft空队列

                db_name = 'db_' + host_for_data

                # 获取要创建的表名称、表前缀
                json_body = []
                for data_group in data_group_list:
                    if re.findall(data_group_regex, data_group): # 找到目标数据组（比如CPU统计，内存统计报告等)
                        row_list = data_group.split('\n')  # 获取组里的每条数据 形如 18:39:11        CPU      %usr     %nice      %sys   %iowait    %steal      %irq     %soft    %guest     %idle
                        row_item_list = re.split('\s+', row_list[0]) # 获取由空格隔开的每项数据
                        if row_item_list:
                            if  row_item_list[1:2] == ['CPU']:
                                flag = 'cpu'
                                field_list = row_item_list[2:] # 获取表字段
                            elif row_item_list[1:2] == ['CPU_LIMIT']: # 数据源：docker 容器
                                flag = 'cpu_limit'
                                field_list = row_item_list[2:]
                            elif row_item_list[1:2] == ['mem_usage']: # 数据源：docker 容器
                                flag = 'mem_swap'
                                field_list = row_item_list[1:]
                            elif row_item_list[1:2] == ['mem_failcnt/s']: # 数据源：docker 容器
                                flag = 'mem_swap_failcnt'
                                field_list = row_item_list[1:]
                            elif row_item_list[1:2] == ['kbmemfree']:
                                flag = 'mem'
                                field_list = row_item_list[1:]
                            elif row_item_list[1:2] == ['proc/s']:
                                flag = 'proc'
                                field_list = row_item_list[1:]
                            elif row_item_list[1:2] == ['proc/s']:
                                flag = 'proc'
                                field_list = row_item_list[1:]
                            elif row_item_list[1:2] == ['pswpin/s']:
                                flag = 'swap'
                                field_list = row_item_list[1:]
                            elif row_item_list[1:2] == ['tps']:
                                flag = 'deviotps'
                                field_list = row_item_list[1:]
                            elif row_item_list[1:2] == ['pgpgin/s']:
                                flag = 'paging'
                                field_list = row_item_list[1:]
                            elif row_item_list[1:2] == ['kbswpfree']:
                                flag = 'swapspace'
                                field_list = row_item_list[1:]
                            elif row_item_list[1:2] == ['runq-sz']:
                                flag = 'queue'
                                field_list = row_item_list[1:]
                            elif row_item_list[1:2] == ['DEV']:
                                flag = 'dev'
                                field_list = row_item_list[2:]
                            elif row_item_list[2:3] == ['rxpck/s']:
                                flag = 'netdev'
                                field_list = row_item_list[2:]
                            elif row_item_list[2:3] == ['rxerr/s']:
                                flag = 'enetdev'
                                field_list = row_item_list[2:]
                            else:
                                continue
                    # elif re.findall('Average\s+', data_group):
                    #     row_list = data_group.split('\n')  # 获取组里的每条数据 形如 Average        CPU      %usr     %nice      %sys   %iowait    %steal      %irq     %soft    %guest     %idle
                    #     row_item_list = re.split('\s+', row_list[0]) # 获取由空格隔开的每项数据
                    #     if row_item_list:
                    #         if  row_item_list[1:2] == ['CPU']:
                    #             flag = 'cpu_avg'
                    #             field_list = row_item_list[2:] # 获取表字段
                    #         elif row_item_list[1:2] == ['proc/s']:
                    #             flag = 'proc_avg'
                    #             field_list = row_item_list[1:]
                    #         elif row_item_list[1:2] == ['pswpin/s']:
                    #             flag = 'swap_avg'
                    #             field_list = row_item_list[1:]
                    #         elif row_item_list[1:2] == ['tps']:
                    #             flag = 'deviotps_avg'
                    #             field_list = row_item_list[1:]
                    #         elif row_item_list[1:2] == ['pgpgin/s']:
                    #             flag = 'paging_avg'
                    #             field_list = row_item_list[1:]
                    #         elif row_item_list[1:2] == ['kbmemfree']:
                    #             flag = 'mem_avg'
                    #             field_list = row_item_list[1:]
                    #         elif row_item_list[1:2] == ['kbswpfree']:
                    #             flag = 'swapspace_avg'
                    #             field_list = row_item_list[1:]
                    #         elif row_item_list[1:2] == ['runq-sz']:
                    #             flag = 'queue_avg'
                    #             field_list = row_item_list[1:]
                    #         elif row_item_list[1:2] == ['DEV']:
                    #             flag = 'dev_avg'
                    #             field_list = row_item_list[2:]
                    #         elif row_item_list[2:3] == ['rxpck/s']:
                    #             flag = 'netdev_avg'
                    #             field_list = row_item_list[2:]
                    #         elif row_item_list[2:3] == ['rxerr/s']:
                    #             flag = 'enetdev_avg'
                    #             field_list = row_item_list[2:]
                    #         else:
                    #             continue

                        for row in row_list[1:]:
                            row_item_list = re.split('\s+', row.strip()) # 获取由空格隔开的每项数据
                            data_time = row_item_list[0]
                            if run_time == '': # 数据来源docker容器 data_time 格式形如 22:42:30-2018-06-18
                                data_time, runtime = data_time.split('@')
                                date_time = '%s %s' % (runtime, data_time)
                            else:
                                date_time = '%s %s' % (run_time, data_time)
                            timetuple = time.strptime(date_time, '%Y-%m-%d %H:%M:%S')
                            second_for_localtime_utc = int(time.mktime(timetuple)) - 8 * 3600 # UTC时间（秒）
                            timetuple = time.localtime(second_for_localtime_utc)
                            date_for_data = time.strftime('%Y-%m-%d', timetuple)
                            time_for_data = time.strftime('%H:%M:%S', timetuple)
                            datetime_for_data = '%sT%sZ' % (date_for_data, time_for_data)

                            if flag in ('cpu', 'cpu_limit', 'dev', 'netdev', 'enetdev'):
                                measurement = flag + '_' + row_item_list[1]
                                row_item_list = row_item_list[2:]
                            else:
                                measurement = flag
                                row_item_list = row_item_list[1:]

                            temp_dic = {}
                            for i in range(0, len(field_list)):
                                temp_dic[field_list[i]] = float(row_item_list[i])  # 必须是数值，否则鼠标和移动到grafana图表曲线时，不显示对应时间点的数据

                            # json_body = [
                            #     {
                            #         "measurement": measurement,
                            #         "tags": {
                            #             "stuid": tag
                            #         },
                            #         "time": datetime_for_data,
                            #         "fields":temp_dic
                            #     }
                            # ]

                            # result = self.influx_db_client.write_points(json_body, database=db_name)
                            # if not result:
                            #     logger.error('采集性能数据失败-往InfluxDB 写入数据失败')
                            #     file_for_failure.write('监控失败\n')
                            #     file_for_failure.write('往InfluxDB 写入数据失败\n')
                            #     file_for_failure.write('数据库：%s\n 写入失败数据：\n' % (db_name, str(json_body)))
                            #     file_for_failure.flush()
                            # else:
                            #     logger.info('往数据库:db_%s 成功写入数据' % host_for_data)
                                # logger.info(json_body)

                            json_body.append({
                                    "measurement": measurement,
                                    "tags": {
                                        "stuid": tag
                                    },
                                    "time": datetime_for_data,
                                    "fields":temp_dic
                            })

                if json_body:
                    result = self.influx_db_client.write_points(json_body, database=db_name)
                    if not result:
                        logger.error('采集性能数据失败-往InfluxDB 写入数据失败')
                        file_for_failure.write('监控失败\n')
                        file_for_failure.write('往InfluxDB 写入数据失败\n')
                        file_for_failure.write('数据库：%s\n 写入失败数据：%s\n' % (db_name, str(json_body)))
                        file_for_failure.flush()
                    else:
                        logger.info('往数据库:db_%s 成功写入数据' % host_for_data)
                else:
                    logger.error('获取influxdb json数据为空')

        file_for_failure = open('./result/result_for_monitor_failure.txt', 'w', encoding='utf-8') # 用于记录下执行监控失败记录
        ssh_client_dic = {} # 用于存放每台主机对应的ssh_client对象
        bastion_host_list = list(self.bashtion_target_host_dic.keys()) # 堡垒机ip列表
        target_host_set_visited_via_bastion = set() # 存放需要通过堡垒机访问的目标主机
        for host in self.target_host_set:
            if host in bastion_host_list:
                target_host_list_visited_via_bastion = self.bashtion_target_host_dic[host]
            else:
                target_host_list_visited_via_bastion=['']

            port, username, password =int(self.host_config[host]['port']),  self.host_config[host]['username'], self.host_config[host]['password']
            retry = 0 # 失败重连次数
            retry_max = 1  # 失败重连最大尝试次数
            for host_visited_via_bastion in target_host_list_visited_via_bastion:
                ssh_client = MySSHClient()
                while retry <= retry_max:
                    logger.info('正在远程访问主机：%s | %s' % (host, host_visited_via_bastion))
                    result = ssh_client.connect(host, port, username, password, host_visited_via_bastion)
                    if not result[0] and retry == retry_max :
                        logger.info('远程访问主机：%s | %s 失败: %s' % (host, host_visited_via_bastion, result[1]))
                        ssh_client.close()
                        file_for_failure.write('监控失败\n')
                        file_for_failure.write('主机信息：\nhost： %s  port：%s username：%s password：%s host_visited_via_bastion：%s ' % (host, port, username, password, host_visited_via_bastion))
                        file_for_failure.write('失败原因：\n连接失败：%s\n\n' % result[1])
                        file_for_failure.flush()
                        logger.error('远程连接主机(%s)失败，提前退出程序：%s' % (host, result[1]))
                        exit()
                    elif result[0]:
                        if host_visited_via_bastion == '':
                            logger.info('正在远程访问主机：%s 成功' % host)
                            ssh_client_dic[host] = ssh_client
                        else:
                            logger.info('通过堡垒机：%s 访问目标机：%s 成功'  % (host, host_visited_via_bastion))
                            ssh_client_dic[host_visited_via_bastion] = ssh_client
                            target_host_set_visited_via_bastion.add(host_visited_via_bastion)
                        break
                    retry += 1
                try:
                    if host_visited_via_bastion == '':
                        db_name =  'db_'+ host
                    else:
                        db_name =  'db_' + host_visited_via_bastion
                    if db_name not in self.database_list:
                        logger.info('influxdb数据库 %s 不存在，正在创建数据库: %s' % (db_name, db_name))
                        self.influx_db_client.create_database(db_name)
                except Exception as e:
                    logger.error('%s' % e)
                    exit()


        logger.info('开始执行监控')
        thread_group1_list = []
        target_host_set = self.target_host_set - set(list(self.bashtion_target_host_dic.keys()))

        for host in target_host_set:
            ssh_client = ssh_client_dic[host]
            # # 多线程执行性能监控
            thread = threading.Thread(target=monitor, args=(host, ssh_client))
            thread.start()
            thread_group1_list.append(thread)

        for host_visited_via_bastion in target_host_set_visited_via_bastion:
            ssh_client = ssh_client_dic[host_visited_via_bastion]
            # # 多线程执行性能监控
            thread = threading.Thread(target=monitor, args=(host_visited_via_bastion, ssh_client, self.bashtion_target_host_dic[host_visited_via_bastion]))
            thread.start()
            thread_group1_list.append(thread)

        run_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) # 作为influxdb tag的一部分
        second_for_localtime = int(time.mktime(time.localtime()))
        tag_prefix = str(315360000000 - second_for_localtime) # 加前缀，筛选stuid时，让最新的运行结果排在最上面 # 一万年的秒数-目前时间撮的秒数
        tag = tag_prefix + '-' + run_time

        current_active_thread_num = threading.active_count()
        active_thread_group1_list = [t for t in thread_group1_list if t.is_alive()] # 获取监控性采集线程组中还活着的线程
        active_thread_group2_list = [] # 存储解析数据线程组中还活着的线程
        second_for_localtime1 = time.mktime(time.localtime()) # UTC时间（秒）

        while current_active_thread_num != 1:
            active_thread_group2_list = [t for t in active_thread_group2_list if t.is_alive()]
            thread_num_to_new = len(active_thread_group1_list) - len(active_thread_group2_list)

            # 根据差值动态创建线程
            for i in range(0, thread_num_to_new):
                thread = threading.Thread(target=parse_data,
                                          args=(tag,))

                thread.start()
                active_thread_group2_list.append(thread)
            current_active_thread_num = threading.active_count()
            active_thread_group1_list = [t for t in thread_group1_list if t.is_alive()]
        second_for_localtime2 = time.mktime(time.localtime()) # UTC时间（秒）
        logger.info('解析完成,耗时(秒)：%s' % (second_for_localtime2-second_for_localtime1))
        self.influx_db_client.close()
        file_for_failure.close()